import os
import logging
from logging import debug, info, error, warning
import numpy as np
import multiprocessing
import coloredlogs, logging
import dip_celery as dip
import time

coloredlogs.install(level='DEBUG', logger=logging.getLogger(),
                    fmt='%(filename)s:%(lineno)d %(message)s',
                    datefmt='%H:%M:%S',
                    )

home = os.getenv('HOME', '')
database_root = home + '/data/yunnan'
files = os.listdir(database_root)
num_thread = multiprocessing.cpu_count()
runningTasks=[]

ind =0
for f in files:
    ind += 1
    f = database_root + '/' + f
    debug('adding ' + f)
    while len(runningTasks) >= num_thread:
        removed = False
        for index in range(len(runningTasks)):
            if runningTasks[index].ready():
                finished_task = runningTasks.pop(index)
                finished_task.wait()
                removed = True
                break
        if not removed:
            time.sleep(0.1)
    runningTasks.append(dip.handle_hospital_data.delay(f))


